{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.196171Z",
     "start_time": "2025-09-03T08:49:19.119245Z"
    }
   },
   "source": [
    "import gymnasium as gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.02274187,  0.01775387,  0.02032313,  0.02849803], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.240301Z",
     "start_time": "2025-09-03T08:49:19.203189Z"
    }
   },
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjUsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvWftoOwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAJZlJREFUeJzt3QtwlNXdx/H/5kogJDSE3CRBEOQetICQeqmWSORWqXHGC8VoGXilwAhRxFgEwY6h2Km3YuhMW7EzIIojWFDQGCQUiYCRFIiSGl5aQskF5U0CsQlJ9nnnnM7uZCWBXHnOZr+fmccnzz4nuyfHXH6c2zosy7IEAADAIH52VwAAAOD7CCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDi2BpR169bJtddeKz169JAJEybIwYMH7awOAADw9YDy1ltvSXp6uqxcuVK++OILGTNmjKSkpEhFRYVdVQIAAIZw2PVmgarHZPz48fL73/9eXzudTomPj5dFixbJU089ZUeVAACAIQLseNGLFy9Kfn6+ZGRkuB/z8/OT5ORkycvLu6R8XV2dPlxUmDl37pz07dtXHA7HVas3AABoP9Uncv78eYmLi9N/940LKN988400NjZKdHS0x+Pq+vjx45eUz8zMlFWrVl3FGgIAgK5SUlIi/fv3Ny+gtJXqaVHzVVyqqqokISFBf4FhYWG21g0AALROdXW1ns7Ru3fvK5a1JaBERkaKv7+/lJeXezyurmNiYi4pHxwcrI/vU+GEgAIAgHdpzfQMW1bxBAUFydixYyUnJ8djXom6TkpKsqNKAADAILYN8aghm7S0NBk3bpzcdNNN8tJLL0lNTY088sgjdlUJAAD4ekC577775OzZs7JixQopKyuTG264QXbt2nXJxFkAAOB7bNsHpaOTbMLDw/VkWeagAADQ/f5+8148AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADdP6A8++yz4nA4PI5hw4a579fW1sqCBQukb9++EhoaKqmpqVJeXt7Z1QAAAF6sS3pQRo4cKaWlpe5j37597ntLliyR7du3y5YtWyQ3N1fOnDkj99xzT1dUAwAAeKmALnnSgACJiYm55PGqqir505/+JJs2bZKf/OQn+rHXX39dhg8fLp999plMnDixK6oDAAC8TJf0oHz99dcSFxcngwYNklmzZsmpU6f04/n5+VJfXy/Jycnusmr4JyEhQfLy8lp8vrq6OqmurvY4AABA99XpAWXChAmyYcMG2bVrl2RlZcnJkyfl1ltvlfPnz0tZWZkEBQVJnz59PD4nOjpa32tJZmamhIeHu4/4+PjOrjYAAOjOQzxTpkxxf5yYmKgDy4ABA+Ttt9+WkJCQdj1nRkaGpKenu69VDwohBQCA7qvLlxmr3pLrr79eiouL9byUixcvSmVlpUcZtYqnuTkrLsHBwRIWFuZxAACA7qvLA8qFCxfkxIkTEhsbK2PHjpXAwEDJyclx3y8qKtJzVJKSkrq6KgAAwFeHeJ544gmZMWOGHtZRS4hXrlwp/v7+8sADD+j5I3PmzNHDNREREbonZNGiRTqcsIIHAAB0WUA5ffq0DiPffvut9OvXT2655Ra9hFh9rLz44ovi5+enN2hTq3NSUlLktdde6+xqAAAAL+awLMsSL6MmyareGLWvCvNRAADofn+/eS8eAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAID3B5S9e/fKjBkzJC4uThwOh2zbts3jvmVZsmLFComNjZWQkBBJTk6Wr7/+2qPMuXPnZNasWRIWFiZ9+vSROXPmyIULFzr+1QAAAN8MKDU1NTJmzBhZt25ds/fXrl0rr7zyiqxfv14OHDggvXr1kpSUFKmtrXWXUeGksLBQsrOzZceOHTr0zJs3r2NfCQAA6DYcluryaO8nOxyydetWmTlzpr5WT6V6Vh5//HF54okn9GNVVVUSHR0tGzZskPvvv1+++uorGTFihBw6dEjGjRuny+zatUumTp0qp0+f1p9/JdXV1RIeHq6fW/XCAAAA87Xl73enzkE5efKklJWV6WEdF1WRCRMmSF5enr5WZzWs4woniirv5+ene1yaU1dXp7+opgcAAOi+OjWgqHCiqB6TptS16546R0VFedwPCAiQiIgId5nvy8zM1EHHdcTHx3dmtQEAgGG8YhVPRkaG7g5yHSUlJXZXCQAAeEtAiYmJ0efy8nKPx9W16546V1RUeNxvaGjQK3tcZb4vODhYj1U1PQAAQPfVqQFl4MCBOmTk5OS4H1PzRdTckqSkJH2tzpWVlZKfn+8us3v3bnE6nXquCgAAQEBbP0HtV1JcXOwxMbagoEDPIUlISJDFixfLr3/9axkyZIgOLM8884xemeNa6TN8+HC56667ZO7cuXopcn19vSxcuFCv8GnNCh4AAND9tTmgfP7553LHHXe4r9PT0/U5LS1NLyV+8skn9V4pal8T1VNyyy236GXEPXr0cH/Oxo0bdSiZNGmSXr2Tmpqq904BAADo8D4odmEfFAAAvI9t+6AAAAB0BgIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADvDyh79+6VGTNmSFxcnDgcDtm2bZvH/Ycfflg/3vS46667PMqcO3dOZs2aJWFhYdKnTx+ZM2eOXLhwoeNfDQAA8M2AUlNTI2PGjJF169a1WEYFktLSUvfx5ptvetxX4aSwsFCys7Nlx44dOvTMmzevfV8BAADodgLa+glTpkzRx+UEBwdLTExMs/e++uor2bVrlxw6dEjGjRunH3v11Vdl6tSp8tvf/lb3zAAAAN/WJXNQ9uzZI1FRUTJ06FCZP3++fPvtt+57eXl5eljHFU6U5ORk8fPzkwMHDjT7fHV1dVJdXe1xAACA7qvTA4oa3vnLX/4iOTk58pvf/EZyc3N1j0tjY6O+X1ZWpsNLUwEBARIREaHvNSczM1PCw8PdR3x8fGdXGwAAePMQz5Xcf//97o9Hjx4tiYmJct111+lelUmTJrXrOTMyMiQ9Pd19rXpQCCkAAHRfXb7MeNCgQRIZGSnFxcX6Ws1Nqaio8CjT0NCgV/a0NG9FzWlRK36aHgAAoPvq8oBy+vRpPQclNjZWXyclJUllZaXk5+e7y+zevVucTqdMmDChq6sDAAC64xCP2q/E1RuinDx5UgoKCvQcEnWsWrVKUlNTdW/IiRMn5Mknn5TBgwdLSkqKLj98+HA9T2Xu3Lmyfv16qa+vl4ULF+qhIVbwAAAAxWFZltWWplBzSe64445LHk9LS5OsrCyZOXOmHD58WPeSqMAxefJkee655yQ6OtpdVg3nqFCyfft2vXpHBZpXXnlFQkNDW1UHNQdFTZatqqpiuAcAAC/Rlr/fbQ4oJiCgAADgfdry95v34gEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA73+zQADoLOqdNoo/yhLL2XjZcoN+8gsJCO511eoFwH4EFAD2sZxSVXJMrMaGyxZzXuE+gO6HIR4AtnFeoecEgO8ioACwj9Npdw0AGIqAAsA2luUUseyuBQATEVAA2MaiBwVACwgoAGxjWcxBAdA8AgoA21xpeTEA30VAAWAbhngAtISAAsDmIR5myQK4FAEFgH3oQQHQAgIKAHuXGQNAMwgoAGzDJFkALSGgALANk2QBtISAAsA2DPEAaAkBBYB9GOIB0AICCgDbMAcFQEsIKABswxwUAC0hoACwDe/FA6AlBBQAtqEHBUCnBJTMzEwZP3689O7dW6KiomTmzJlSVFTkUaa2tlYWLFggffv2ldDQUElNTZXy8nKPMqdOnZJp06ZJz5499fMsXbpUGhoa2lIVAN0loFhsdQ+ggwElNzdXh4/PPvtMsrOzpb6+XiZPniw1NTXuMkuWLJHt27fLli1bdPkzZ87IPffc477f2Niow8nFixdl//798sYbb8iGDRtkxYoVbakKgO7AauSdeAA0y2FZ7f/ny9mzZ3UPiAoit912m1RVVUm/fv1k06ZNcu+99+oyx48fl+HDh0teXp5MnDhRdu7cKdOnT9fBJTo6WpdZv369LFu2TD9fUFDQFV+3urpawsPD9euFhYW1t/oAbPbt1wflfz/5s+pKuWy5MbNfkKCe4VetXgC6Rlv+fndoDop6ASUiIkKf8/Pzda9KcnKyu8ywYcMkISFBBxRFnUePHu0OJ0pKSoqudGFhYbOvU1dXp+83PQB4PzZqA9DpAcXpdMrixYvl5ptvllGjRunHysrKdA9Inz59PMqqMKLuuco0DSeu+657Lc19UYnLdcTHx7e32gAMwj4oADo9oKi5KMeOHZPNmzdLV8vIyNC9Na6jpKSky18TQNcjoABoSYC0w8KFC2XHjh2yd+9e6d+/v/vxmJgYPfm1srLSoxdFreJR91xlDh486PF8rlU+rjLfFxwcrA8A3UtD7YUrlvEPChGHOK5KfQB4aQ+Kmk+rwsnWrVtl9+7dMnDgQI/7Y8eOlcDAQMnJyXE/ppYhq2XFSUlJ+lqdjx49KhUVFe4yakWQmiwzYsSIjn9FALzGueKDV5wg22fgjeIXeOXJ8wB8uAdFDeuoFTrvvfee3gvFNWdEzQsJCQnR5zlz5kh6erqeOKtCx6JFi3QoUSt4FLUsWQWR2bNny9q1a/VzLF++XD83vSQAvs/hUP+OogcF8DVtCihZWVn6fPvtt3s8/vrrr8vDDz+sP37xxRfFz89Pb9CmVt+oFTqvvfaau6y/v78eHpo/f74OLr169ZK0tDRZvXp153xFALphQAHgazq0D4pd2AcF6B6Ovf2s/Of/zly2TL8Rt0v8xFTxD6SHFfB2V20fFADoag4/f9WNYnc1AFxlBBQARnP4sYYH8EUEFABGczj87a4CABsQUAAYzeHHrynAF/GTD8BoLDMGfBMBBYDZ1CRZAD6HgALAaA61gocOFMDnEFAAeMEkWRIK4GsIKADMxiRZwCfxkw/AaGx1D/gmfvIBGI1lxoBv4icfgNlUDwpb3QM+h4ACwPj34iGeAL6HgALAaMxBAXwTP/kAjMZOsoBvIqAAMBuTZAGfxE8+APN7UOhAAXwOAQWA0VhmDPgmfvIBGI2t7gHfREABYDSHH+EE8EUEFABm0z0oAHwNAQWALSzLavUkWQc7yQI+h4ACwBaW5ZTWRRQAvoiAAsAeugeFiAKgeQQUALb1oJBPALSEgALAHiqgkFAAtICAAsAWllMFFABoHgEFgH2TZFu5kgeA7yGgALAH4QTAZRBQANjCcjYSUgC0iIACwL5VPEySBdAZASUzM1PGjx8vvXv3lqioKJk5c6YUFRV5lLn99tv1ro9Nj0cffdSjzKlTp2TatGnSs2dP/TxLly6VhoaGtlQFgLfTAQUAmhcgbZCbmysLFizQIUUFiqefflomT54sX375pfTq1ctdbu7cubJ69Wr3tQoiLo2NjTqcxMTEyP79+6W0tFQeeughCQwMlOeff74t1QHgxf47QZYeFACdEFB27drlcb1hwwbdA5Kfny+33XabRyBRAaQ5H330kQ40H3/8sURHR8sNN9wgzz33nCxbtkyeffZZCQoKakuVAHjxMmOmoADokjkoVVVV+hwREeHx+MaNGyUyMlJGjRolGRkZ8t1337nv5eXlyejRo3U4cUlJSZHq6mopLCxs9nXq6ur0/aYHAC/HHBQAndWD0pTT6ZTFixfLzTffrIOIy4MPPigDBgyQuLg4OXLkiO4ZUfNU3n33XX2/rKzMI5wormt1r6W5L6tWrWpvVQEYiK3uAXRJQFFzUY4dOyb79u3zeHzevHnuj1VPSWxsrEyaNElOnDgh1113XbteS/XCpKenu69VD0p8fHx7qw7AAKziAdDpQzwLFy6UHTt2yCeffCL9+/e/bNkJEyboc3FxsT6ruSnl5eUeZVzXLc1bCQ4OlrCwMI8DgJdzEk4AdFJAUbPuVTjZunWr7N69WwYOHHjFzykoKNBn1ZOiJCUlydGjR6WiosJdJjs7W4eOESNGtKU6ALyYZbFRG4BOGuJRwzqbNm2S9957T++F4pozEh4eLiEhIXoYR92fOnWq9O3bV89BWbJkiV7hk5iYqMuqZckqiMyePVvWrl2rn2P58uX6uVVPCQAfWsVjdyUAdI8elKysLL1yR23GpnpEXMdbb72l76slwmr5sAohw4YNk8cff1xSU1Nl+/bt7ufw9/fXw0PqrHpTfv7zn+t9UJrumwLABzAHBUBn9aBc6Z1H1cRVtZnblahVPh988EFbXhpAN6N/n5BPALSA9+IBYAtW8QC4HAIKAHs4eS8eAC0joACwh96ojR4UAM0joACwbYiHeAKgJQQUALbg3YwBXA4BBYAtKv9ZIA3/uXDZMr1jh0hwWN+rVicA5iCgALCF5Wy4Yg+Kwz9Q/eeq1QmAOQgoAMzl8BOHw+5KALADAQWAsRwO9SuKhAL4IgIKAGM5VPcJXSiATyKgADB7iIceFMAnEVAAGIseFMB3EVAAmEvPQQHgi/jpB2Ash5/jv70oAHwOAQWAwVjFA/gqAgoAYzn8/JiDAvgoAgoAYzFJFvBdBBQA5mKZMeCzCCgAzKV7UOyuBAA7BNjyqgC8mmVZ0tjY2KHncDqtVryQSGOjU6RBvbFg+/j7+7MSCPBCBBQAbXb69GkZNGhQh55jxUO3yV03XXfZMuv/8Af5w/T/kZra+naHk/Pnz0tgYGA7awnALgQUAO3S0IFeDcWynFd+jUanXKyvb/drOZ1Xfg0AZiKgALBVo+Uv5XXXynfOMD2mE+pfKdFB/9TTT9QwkNWKkSAA3Q8BBYBtVPj4ovpOqW6IlHqrhw4oQX61UnFxgCT2ztU9IGq+CwDfQ0ABYAun5ScHq6fLufpYj91i65y95EzdEHGIJY3WEXpQAB/FMmMAtjh64ceXhBMXS/zkdN1QKa4ZLZZaygPA5xBQANjocst/HaLmuNKDAvgmAgoAYzkt5qAAvoqAAsBY9KAAvouAAsAWI0P3SVjA2f9uF3sJS2KC/lcSehxhDgrgo9oUULKysiQxMVHCwsL0kZSUJDt37nTfr62tlQULFkjfvn0lNDRUUlNTpby83OM5Tp06JdOmTZOePXtKVFSULF26tMMbPgHwPgGOevlR+FYJDzgrAY461V+ij0BHrUQF/Utu6P2xOKwGelAAH9WmZcb9+/eXNWvWyJAhQ/S48BtvvCF33323HD58WEaOHClLliyR999/X7Zs2SLh4eGycOFCueeee+TTTz/Vn6/eu0OFk5iYGNm/f7+UlpbKQw89pLehfv7557vqawRgoPyiUqmrb5RG67fy79ohcqHxB3ppce+Ab6V/j39IiYgUlXxjdzUB2MRhdXAGWkREhLzwwgty7733Sr9+/WTTpk36Y+X48eMyfPhwycvLk4kTJ+relunTp8uZM2ckOjpal1m/fr0sW7ZMzp49K0FBQa16zerqah2AHn744VZ/DoDOU1NTIxs3bhTTqTcJnDNnjvj5MZoNmODixYuyYcMGqaqq0iMxXbJRm+oNUT0l6heVGurJz8+X+vp6SU5OdpcZNmyYJCQkuAOKOo8ePdodTpSUlBSZP3++FBYWyo033tjsa9XV1emjaUBRZs+erYeSAFxdaujWWwLKI488IgEB7EkJmODChQs6oLRGm39qjx49qgOJmm+iwsHWrVtlxIgRUlBQoHsz+vTp41FehZGysjL9sTo3DSeu+657LcnMzJRVq1Zd8vi4ceOumMAAdL6SEjUA4x3Gjx/PuxkDhnB1MLRGm/s9hw4dqsPIgQMHdM9HWlqafPnll9KVMjIydHeQ6/CmX44AAKDt2tyDonpJBg8erD8eO3asHDp0SF5++WW577779NhSZWWlRy+K6gpWk2IVdT548KDH87lW+bjKNCc4OFgfAADAN3R45ph6t1E1P0SFFdWNmpOT475XVFSklxWrISFFndUQUUVFhbtMdna2HqZRw0QAAABt7kFRQy1TpkzRE1/Pnz+vV+zs2bNHPvzwQ72qRs2WT09P1yt7VOhYtGiRDiVqgqwyefJkHUTU5Na1a9fqeSfLly/Xe6fQQwIAANoVUFTPh9q3RO1fogKJ2rRNhZM777xT33/xxRf1cj61QZvqVVErdF577TX35/v7+8uOHTv03BUVXHr16qXnsKxevbot1QAAAN1ch/dBsYNrH5TWrKMG0PnURHXVk2o69Q8mteKQVTyA9/39ZvciAABgHAIKAAAwDgEFAAAYh4ACAACMwxtUAGizkJAQmTlzpnjDJFn1fjwAvA8BBUCbRUZG6vfhAoCuwhAPAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAADg3QElKytLEhMTJSwsTB9JSUmyc+dO9/3bb79dHA6Hx/Hoo496PMepU6dk2rRp0rNnT4mKipKlS5dKQ0ND531FAADA6wW0pXD//v1lzZo1MmTIELEsS9544w25++675fDhwzJy5EhdZu7cubJ69Wr356gg4tLY2KjDSUxMjOzfv19KS0vloYceksDAQHn++ec78+sCAABezGGppNEBERER8sILL8icOXN0D8oNN9wgL730UrNlVW/L9OnT5cyZMxIdHa0fW79+vSxbtkzOnj0rQUFBrXrN6upqCQ8Pl6qqKt2TAwAAzNeWv9/tnoOiekM2b94sNTU1eqjHZePGjRIZGSmjRo2SjIwM+e6779z38vLyZPTo0e5woqSkpOgKFxYWtvhadXV1ukzTAwAAdF9tGuJRjh49qgNJbW2thIaGytatW2XEiBH63oMPPigDBgyQuLg4OXLkiO4ZKSoqknfffVffLysr8wgniuta3WtJZmamrFq1qq1VBQAAvhJQhg4dKgUFBbp75p133pG0tDTJzc3VIWXevHnucqqnJDY2ViZNmiQnTpyQ6667rt2VVD0x6enp7mvVgxIfH9/u5wMAAGZr8xCPmicyePBgGTt2rO7ZGDNmjLz88svNlp0wYYI+FxcX67OaHFteXu5RxnWt7rUkODjYvXLIdQAAgO6rw/ugOJ1OPUekOaqnRVE9KYoaGlJDRBUVFe4y2dnZOnC4hokAAAAC2jrUMmXKFElISJDz58/Lpk2bZM+ePfLhhx/qYRx1PXXqVOnbt6+eg7JkyRK57bbb9N4pyuTJk3UQmT17tqxdu1bPO1m+fLksWLBA95IAAAC0OaCong+1b4nav0QtE1LBQ4WTO++8U0pKSuTjjz/WS4zVyh41RyQ1NVUHEBd/f3/ZsWOHzJ8/X/em9OrVS89habpvCgAAQIf3QbED+6AAAOB9rso+KAAAAF2FgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGCdAvJBlWfpcXV1td1UAAEAruf5uu/6Od7uAcv78eX2Oj4+3uyoAAKAdf8fDw8MvW8ZhtSbGGMbpdEpRUZGMGDFCSkpKJCwszO4qeXWaVUGPduw42rLz0Jadg3bsPLRl51CRQ4WTuLg48fPz6349KOqLuuaaa/TH6huFb5aOox07D23ZeWjLzkE7dh7asuOu1HPiwiRZAABgHAIKAAAwjtcGlODgYFm5cqU+o/1ox85DW3Ye2rJz0I6dh7a8+rxykiwAAOjevLYHBQAAdF8EFAAAYBwCCgAAMA4BBQAAGMcrA8q6devk2muvlR49esiECRPk4MGDdlfJOHv37pUZM2bo3focDods27bN476aG71ixQqJjY2VkJAQSU5Olq+//tqjzLlz52TWrFl6U6I+ffrInDlz5MKFC+JLMjMzZfz48dK7d2+JioqSmTNn6l2Mm6qtrZUFCxZI3759JTQ0VFJTU6W8vNyjzKlTp2TatGnSs2dP/TxLly6VhoYG8RVZWVmSmJjo3uQqKSlJdu7c6b5PG7bfmjVr9M/44sWL3Y/Rnq3z7LPP6rZregwbNsx9n3a0meVlNm/ebAUFBVl//vOfrcLCQmvu3LlWnz59rPLycrurZpQPPvjA+tWvfmW9++67apWWtXXrVo/7a9asscLDw61t27ZZf//7362f/vSn1sCBA63//Oc/7jJ33XWXNWbMGOuzzz6z/va3v1mDBw+2HnjgAcuXpKSkWK+//rp17Ngxq6CgwJo6daqVkJBgXbhwwV3m0UcfteLj462cnBzr888/tyZOnGj96Ec/ct9vaGiwRo0aZSUnJ1uHDx/W/28iIyOtjIwMy1f89a9/td5//33rH//4h1VUVGQ9/fTTVmBgoG5XhTZsn4MHD1rXXnutlZiYaD322GPux2nP1lm5cqU1cuRIq7S01H2cPXvWfZ92tJfXBZSbbrrJWrBggfu6sbHRiouLszIzM22tl8m+H1CcTqcVExNjvfDCC+7HKisrreDgYOvNN9/U119++aX+vEOHDrnL7Ny503I4HNa///1vy1dVVFTodsnNzXW3m/pDu2XLFneZr776SpfJy8vT1+qXlp+fn1VWVuYuk5WVZYWFhVl1dXWWr/rBD35g/fGPf6QN2+n8+fPWkCFDrOzsbOvHP/6xO6DQnm0LKOofYc2hHe3nVUM8Fy9elPz8fD0c0fR9edR1Xl6erXXzJidPnpSysjKPdlTvjaCGy1ztqM5qWGfcuHHuMqq8au8DBw6Ir6qqqtLniIgIfVbfj/X19R5tqbqIExISPNpy9OjREh0d7S6TkpKi33yssLBQfE1jY6Ns3rxZampq9FAPbdg+auhBDS00bTeF9mwbNbSthsIHDRqkh7TVkI1CO9rPq94s8JtvvtG/3Jp+Myjq+vjx47bVy9uocKI0146ue+qsxlObCggI0H+YXWV8jXoXbTXOf/PNN8uoUaP0Y6otgoKCdJi7XFs219aue77i6NGjOpCocX01nr9161b9juQFBQW0YRupgPfFF1/IoUOHLrnH92TrqX+UbdiwQYYOHSqlpaWyatUqufXWW+XYsWO0owG8KqAAdv+LVf3i2rdvn91V8Urqj4AKI6oX6p133pG0tDTJzc21u1pep6SkRB577DHJzs7WCwXQflOmTHF/rCZxq8AyYMAAefvtt/XiAdjLq4Z4IiMjxd/f/5JZ1Oo6JibGtnp5G1dbXa4d1bmiosLjvpqZrlb2+GJbL1y4UHbs2CGffPKJ9O/f3/24ags19FhZWXnZtmyurV33fIX61+jgwYNl7NixenXUmDFj5OWXX6YN20gNPaifzR/+8Ie6V1MdKui98sor+mP1L3jas31Ub8n1118vxcXFfF8awM/bfsGpX245OTke3e7qWnUdo3UGDhyof3iatqMaM1VzS1ztqM7qB1P9MnTZvXu3bm/1rwxfoeYYq3CihiPU16/arin1/RgYGOjRlmoZshrHbtqWanijaeBT//pVy23VEIevUt9LdXV1tGEbTZo0SbeF6o1yHWqumJo/4fqY9mwftY3CiRMn9PYLfF8awPLCZcZqtcmGDRv0SpN58+bpZcZNZ1HjvzP81bI3daj/zb/73e/0x//617/cy4xVu7333nvWkSNHrLvvvrvZZcY33nijdeDAAWvfvn16xYCvLTOeP3++Xo69Z88ej6WI3333ncdSRLX0ePfu3XopYlJSkj6+vxRx8uTJeqnyrl27rH79+vnUUsSnnnpKr3w6efKk/n5T12pF2EcffaTv04Yd03QVj0J7ts7jjz+uf7bV9+Wnn36qlwurZcJqtZ5CO9rL6wKK8uqrr+pvGrUfilp2rPbpgKdPPvlEB5PvH2lpae6lxs8884wVHR2tA9+kSZP0/hRNffvttzqQhIaG6mVzjzzyiA4+vqS5NlSH2hvFRYW6X/7yl3rZbM+ePa2f/exnOsQ09c9//tOaMmWKFRISon8Bql+M9fX1lq/4xS9+YQ0YMED/zKpf4Or7zRVOFNqwcwMK7dk69913nxUbG6u/L6+55hp9XVxc7L5PO9rLof5jdy8OAACA185BAQAAvoGAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAAAxzf8DmEDKAWSQlPYAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.253188Z",
     "start_time": "2025-09-03T08:49:19.249577Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "#定义模型\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model_td = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 1),\n",
    ")\n",
    "\n",
    "model(torch.randn(2, 4)), model_td(torch.randn(2, 4))"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.2899, 0.7101],\n",
       "         [0.4868, 0.5132]], grad_fn=<SoftmaxBackward0>),\n",
       " tensor([[-0.7218],\n",
       "         [ 0.0521]], grad_fn=<AddmmBackward0>))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.272874Z",
     "start_time": "2025-09-03T08:49:19.270285Z"
    }
   },
   "source": [
    "import random\n",
    "\n",
    "\n",
    "#得到一个动作\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 4)\n",
    "    #[1, 4] -> [1, 2]\n",
    "    prob = model(state)\n",
    "\n",
    "    #根据概率选择一个动作\n",
    "    action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]\n",
    "\n",
    "    return action\n",
    "\n",
    "\n",
    "get_action([1, 2, 3, 4])"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.303818Z",
     "start_time": "2025-09-03T08:49:19.298409Z"
    }
   },
   "source": [
    "def get_data():\n",
    "    states = []\n",
    "    rewards = []\n",
    "    actions = []\n",
    "    next_states = []\n",
    "    overs = []\n",
    "\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "        #记录数据样本\n",
    "        states.append(state)\n",
    "        rewards.append(reward)\n",
    "        actions.append(action)\n",
    "        next_states.append(next_state)\n",
    "        overs.append(over)\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    #[b, 4]\n",
    "    states = torch.FloatTensor(states).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    rewards = torch.FloatTensor(rewards).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    actions = torch.LongTensor(actions).reshape(-1, 1)\n",
    "    #[b, 4]\n",
    "    next_states = torch.FloatTensor(next_states).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    overs = torch.LongTensor(overs).reshape(-1, 1)\n",
    "\n",
    "    return states, rewards, actions, next_states, overs\n",
    "\n",
    "\n",
    "get_data()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[-2.9446e-02,  1.4454e-02,  7.3168e-03, -4.4698e-03],\n",
       "         [-2.9157e-02,  2.0947e-01,  7.2274e-03, -2.9484e-01],\n",
       "         [-2.4967e-02,  1.4246e-02,  1.3307e-03,  1.1832e-04],\n",
       "         [-2.4682e-02,  2.0935e-01,  1.3330e-03, -2.9214e-01],\n",
       "         [-2.0495e-02,  4.0445e-01, -4.5098e-03, -5.8441e-01],\n",
       "         [-1.2406e-02,  2.0939e-01, -1.6198e-02, -2.9315e-01],\n",
       "         [-8.2184e-03,  4.0474e-01, -2.2061e-02, -5.9089e-01],\n",
       "         [-1.2352e-04,  2.0994e-01, -3.3879e-02, -3.0524e-01],\n",
       "         [ 4.0752e-03,  4.0552e-01, -3.9984e-02, -6.0841e-01],\n",
       "         [ 1.2186e-02,  2.1098e-01, -5.2152e-02, -3.2859e-01],\n",
       "         [ 1.6405e-02,  4.0681e-01, -5.8724e-02, -6.3725e-01],\n",
       "         [ 2.4541e-02,  2.1255e-01, -7.1469e-02, -3.6362e-01],\n",
       "         [ 2.8793e-02,  4.0861e-01, -7.8741e-02, -6.7796e-01],\n",
       "         [ 3.6965e-02,  2.1467e-01, -9.2300e-02, -4.1107e-01],\n",
       "         [ 4.1258e-02,  4.1097e-01, -1.0052e-01, -7.3136e-01],\n",
       "         [ 4.9477e-02,  6.0733e-01, -1.1515e-01, -1.0539e+00],\n",
       "         [ 6.1624e-02,  8.0377e-01, -1.3623e-01, -1.3804e+00],\n",
       "         [ 7.7699e-02,  6.1059e-01, -1.6384e-01, -1.1332e+00],\n",
       "         [ 8.9911e-02,  4.1794e-01, -1.8650e-01, -8.9610e-01],\n",
       "         [ 9.8270e-02,  6.1504e-01, -2.0442e-01, -1.2411e+00]]),\n",
       " tensor([[1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.]]),\n",
       " tensor([[1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [1]]),\n",
       " tensor([[-2.9157e-02,  2.0947e-01,  7.2274e-03, -2.9484e-01],\n",
       "         [-2.4967e-02,  1.4246e-02,  1.3307e-03,  1.1832e-04],\n",
       "         [-2.4682e-02,  2.0935e-01,  1.3330e-03, -2.9214e-01],\n",
       "         [-2.0495e-02,  4.0445e-01, -4.5098e-03, -5.8441e-01],\n",
       "         [-1.2406e-02,  2.0939e-01, -1.6198e-02, -2.9315e-01],\n",
       "         [-8.2184e-03,  4.0474e-01, -2.2061e-02, -5.9089e-01],\n",
       "         [-1.2352e-04,  2.0994e-01, -3.3879e-02, -3.0524e-01],\n",
       "         [ 4.0752e-03,  4.0552e-01, -3.9984e-02, -6.0841e-01],\n",
       "         [ 1.2186e-02,  2.1098e-01, -5.2152e-02, -3.2859e-01],\n",
       "         [ 1.6405e-02,  4.0681e-01, -5.8724e-02, -6.3725e-01],\n",
       "         [ 2.4541e-02,  2.1255e-01, -7.1469e-02, -3.6362e-01],\n",
       "         [ 2.8793e-02,  4.0861e-01, -7.8741e-02, -6.7796e-01],\n",
       "         [ 3.6965e-02,  2.1467e-01, -9.2300e-02, -4.1107e-01],\n",
       "         [ 4.1258e-02,  4.1097e-01, -1.0052e-01, -7.3136e-01],\n",
       "         [ 4.9477e-02,  6.0733e-01, -1.1515e-01, -1.0539e+00],\n",
       "         [ 6.1624e-02,  8.0377e-01, -1.3623e-01, -1.3804e+00],\n",
       "         [ 7.7699e-02,  6.1059e-01, -1.6384e-01, -1.1332e+00],\n",
       "         [ 8.9911e-02,  4.1794e-01, -1.8650e-01, -8.9610e-01],\n",
       "         [ 9.8270e-02,  6.1504e-01, -2.0442e-01, -1.2411e+00],\n",
       "         [ 1.1057e-01,  8.1211e-01, -2.2925e-01, -1.5903e+00]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1]]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.322022Z",
     "start_time": "2025-09-03T08:49:19.318414Z"
    }
   },
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:19.336548Z",
     "start_time": "2025-09-03T08:49:19.334144Z"
    }
   },
   "source": [
    "#优势函数\n",
    "def get_advantages(deltas):\n",
    "    advantages = []\n",
    "\n",
    "    #反向遍历deltas\n",
    "    s = 0.0\n",
    "    for delta in deltas[::-1]:\n",
    "        s = 0.98 * 0.95 * s + delta\n",
    "        advantages.append(s)\n",
    "\n",
    "    #逆序\n",
    "    advantages.reverse()\n",
    "    return advantages\n",
    "\n",
    "\n",
    "get_advantages(range(5))"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[8.090483997483998, 8.690100963999999, 8.260044, 6.724, 4.0]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "metadata": {
    "executionInfo": {
     "elapsed": 8251,
     "status": "ok",
     "timestamp": 1650011468229,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "BQXVYW2T_DcQ",
    "scrolled": false,
    "ExecuteTime": {
     "end_time": "2025-09-03T08:49:23.997692Z",
     "start_time": "2025-09-03T08:49:19.347770Z"
    }
   },
   "source": [
    "def train():\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "    optimizer_td = torch.optim.Adam(model_td.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #玩N局游戏,每局游戏训练M次\n",
    "    for epoch in range(500):\n",
    "        #玩一局游戏,得到数据\n",
    "        #states -> [b, 4]\n",
    "        #rewards -> [b, 1]\n",
    "        #actions -> [b, 1]\n",
    "        #next_states -> [b, 4]\n",
    "        #overs -> [b, 1]\n",
    "        states, rewards, actions, next_states, overs = get_data()\n",
    "\n",
    "        #计算values和targets\n",
    "        #[b, 4] -> [b, 1]\n",
    "        values = model_td(states)\n",
    "\n",
    "        #[b, 4] -> [b, 1]\n",
    "        targets = model_td(next_states).detach()\n",
    "        targets = targets * 0.98\n",
    "        targets *= (1 - overs)\n",
    "        targets += rewards\n",
    "\n",
    "        #计算优势,这里的advantages有点像是策略梯度里的reward_sum\n",
    "        #只是这里计算的不是reward,而是target和value的差\n",
    "        #[b, 1]\n",
    "        deltas = (targets - values).squeeze(dim=1).tolist()\n",
    "        advantages = get_advantages(deltas)\n",
    "        advantages = torch.FloatTensor(advantages).reshape(-1, 1)\n",
    "\n",
    "        #取出每一步动作的概率\n",
    "        #[b, 2] -> [b, 2] -> [b, 1]\n",
    "        old_probs = model(states)\n",
    "        old_probs = old_probs.gather(dim=1, index=actions)\n",
    "        old_probs = old_probs.detach()\n",
    "\n",
    "        #每批数据反复训练10次\n",
    "        for _ in range(10):\n",
    "            #重新计算每一步动作的概率\n",
    "            #[b, 4] -> [b, 2]\n",
    "            new_probs = model(states)\n",
    "            #[b, 2] -> [b, 1]\n",
    "            new_probs = new_probs.gather(dim=1, index=actions)\n",
    "            new_probs = new_probs\n",
    "\n",
    "            #求出概率的变化\n",
    "            #[b, 1] - [b, 1] -> [b, 1]\n",
    "            ratios = new_probs / old_probs\n",
    "\n",
    "            #计算截断的和不截断的两份loss,取其中小的\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr1 = ratios * advantages\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr2 = torch.clamp(ratios, 0.8, 1.2) * advantages\n",
    "\n",
    "            loss = -torch.min(surr1, surr2)\n",
    "            loss = loss.mean()\n",
    "\n",
    "            #重新计算value,并计算时序差分loss\n",
    "            values = model_td(states)\n",
    "            loss_td = loss_fn(values, targets)\n",
    "\n",
    "            #更新参数\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            optimizer_td.zero_grad()\n",
    "            loss_td.backward()\n",
    "            optimizer_td.step()\n",
    "\n",
    "        if epoch % 50 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, test_result)\n",
    "\n",
    "\n",
    "train()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 18.2\n",
      "50 200.0\n",
      "100 91.0\n",
      "150 199.2\n",
      "200 200.0\n",
      "250 200.0\n",
      "300 200.0\n",
      "350 200.0\n",
      "400 200.0\n",
      "450 200.0\n"
     ]
    }
   ],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "test(play=True)"
   ],
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
